#!/usr/bin/env python
# -*- coding: utf8 -*-

"""
pyspi.py

Bot searching e-mail addresses in website.
It provides also saving searched info in sqlite database
and show results as table in webpage.
"""

import urllib, collections, re, sys
from sqlmgr import SQLMgr
from web import app
from parser import Parser
from config import Config

class Spider(object):
    """
    Main class 
    """
    def __init__(self):
        """
        Spider init
        :parameters:
            url : string
                URL to page, where spider start its spinning
        """
        self.unvisited = collections.deque()
        self.visited = collections.deque()
        self.visitedPages = []
        self.conf = Config()
        
    def __getPage(self, url):
        """
        try get page from url
        :parameters:
            url : string
                url to page
        :return:
            page : string
                downloaded page or empty string if can't do this
            info : httplib.HTTPMessage
                headers info's about page
        """
        try:
            sock = urllib.urlopen(url)
            page = sock.read()
            info = sock.info()
            sock.close()
        except IOError:
            return '',''
        return self.__toUnicode(page), info
    
    def __toUnicode(self,page):
        """
        convert page to UTF-8
        :parameters:
            page : string
                page source to encode to unicode
        :return:
            pageunicode : unicode
                converted page
        """
        charsetList =  re.findall('CHARSET=[\'"]?.*?[\'"]', page, re.I)
        try:
            if charsetList:
                charset = re.split('=', re.sub("['\"]", '',\
                            charsetList[0]))[1]
                return page.decode(charset)
            else:
                try:
                    return page.decode('iso-8859-2')
                except:
                    return ''
        except:
            return ''

    def __pushUnvisited(self, urls):
        """
        push unvisited urls into unvisited deque
        :parameters:
            urls : list
                list of urls
        """
        self.unvisited.extend([ item for item in urls \
            if item not in self.visited ])

    def spin(self):
        """
        control spinning process
        """
        db = SQLMgr()
        db.connect()
        db.create()
        self.unvisited.append(self.conf.startpage)
        print 'Spider is spinning... [CTRL + C to stop]'
        try:
            while self.unvisited and\
            len(self.visitedPages) < self.conf.pages:
                url = self.unvisited.popleft()
                page, info = self.__getPage(url)
                self.visited.append(url)
                if info and page:
                    if 'html' in info.type or 'xml' in info.type:
                        parser = Parser()
                        self.__pushUnvisited(Parser.getUrls(page))
                        db.insert(parser.getInfo(page))
                        self.visitedPages.append(url)
        except KeyboardInterrupt:
            pass
        finally:
            db.close()
            if self.conf.webAutostart:
                print '\nStarting web interface...'
                app.run()
def start():
    tmp = Spider()
    if sys.argv[1:]:
        if '-p' in sys.argv:
            try:
                tmp.conf.pages = int(sys.argv[sys.argv.index('-p')+1])
            except:
                print 'Invalid argument after "-p".\
                 \nNumber of pages must be an integer'
                return
        if '-a' in sys.argv:
            tmp.conf.webAutostart = True
        elif '-w' in sys.argv:
            app.run()
            return
        if '-s' in sys.argv:
            try:
                tmp.conf.startpage = sys.argv[sys.argv.index('-s')+1]
            except:
                print 'Invalid argument after "-s".\
                \nStart page not defined'
                return
        if '-h' in sys.argv or '--help' in sys.argv:
            print """
    PySpi - bot searching email addresses in website
    
    pyspi [-a|-w] [-p :int:] [-s :string:]

    flags:
        a - autostart web server after searching
        h - show this message
        p :int: - number of pages to search
        s :string: - starting web page
        w - only start web server (use database defined in /etc/pyspi.conf
    """
            return
    tmp.spin()

if __name__ == "__main__":
    start()
